# This is a code cell
# This is the place where your code is written
# Any lines beginning with the "#" are not code but descriptions to help you and others know what is being done
# hit shift-Enter together to run this cell

# This is another code cell
# Here we have some code written....it is a 'function' , let's not worry about details of 'functions' at the moment
#  it is just for seeing outputs
# in jupyter notebooks, codes show their outputs below the code cell that was run
def perserverance(text):
    return "so long as you do not stop!"


print(f"{perserverance('It does not matter how slowly you go...')}") 

# below this example code cell an output will appear

# another example

print("this will print a message to the output") 


# print("some text") messages help to a) show result of the code, b) understand what went wrong

# uncomment the line below this if you want to see which packages you already have installed
#  The '!'  character below is used to run this directly in the notebook
# conda is the package manager recommended
# ! conda list 

# I recommend using conda to install install itables in your terminal 
# why in the terminal and not in the code cell? : because conda requires you to input a yes, no to execute the installation,
#  which may or may not show-up in the cell output
# In the terminal type: conda install itables

import pandas as pd # should already be pre-installed with conda

from itables import init_notebook_mode, show #to fully take advantage of the ability to display our data as a table format, let's use the itables library 

init_notebook_mode(all_interactive=True) #After this, any Pandas DataFrame, or Series, is displayed as interactive table, which lets you explore, filter or sort your data

# the following code cell will give you a nice hover highlighting on your tables when used with itables library

%%html
<style>
  .dataTables_wrapper tbody tr:hover {
    background-color: #6495ED; /* Cornflower Blue */
  }
</style>


<!-- #1E3A8A (Dark Blue) -->
<!-- #0D9488 (Teal) -->
<!-- #065F46 (Dark Green) -->
<!-- #4C1D95 (Dark Purple) -->
<!-- #991B1B (Dark Red) -->
<!-- #374151 (Dark Gray) -->
<!-- #B45309 (Deep Orange) -->
<!-- #164E63 (Dark Cyan) -->
<!-- #4A2C2A (Dark Brown) -->
<!-- #831843 (Dark Magenta) -->
<!-- #1E3A8A (Dark Blue ) -->

<!-- Suggested Light Colors for Light Backgrounds -->
<!-- #AED9E0 (Light Blue) -->
<!-- #A7F3D0 (Light Teal) -->
<!-- #D1FAE5 (Light Green) -->
<!-- #DDD6FE (Light Purple) -->
<!-- #FECACA (Light Red) -->
<!-- #E5E7EB (Light Gray) -->
<!-- #FFEDD5 (Light Orange) -->
<!-- #B2F5EA (Light Cyan) -->
<!-- #FED7AA (Light Brown) -->
<!-- #FBCFE8 (Light Magenta) -->

# here we will load a historical US tornado dataset using pandas

# you can give it any name you want...just has to follow python convention for names i.e., so-called snake case
tor_data = pd.read_csv(r".\Data\1950-2022_actual_tornadoes.csv") 

print(tor_data.head()) 

# You can show the contents of this dataset as a table below by simply typing its name and running this code cell OR using print(data_set_name)

# now see how it looks as an interactive table. You can click on the arrows by the colum names to sort them...
tor_data.head() # note that you do not have to use show() to generate an interactive table unless you want to add addtional customizations 

# To read any dataset into python you need to specify:

    # 1. WHAT - name to save the data to 
    # 2. WHO - is going to do the work? i.e., which package is going to do the work?, here we rely on pandas 'pd' package or library 
    # 3. HOW - read_csv(), the specific method that pandas will use 
    # 4. WHERE - is the file? the file is specified by its name with relative path (i.e. '.\') or by its full path, including its extension, in this case .csv is the extension


# (1)     (2)    (3)             (4) actual file name and its path is always in quotes
tor_data = pd.read_csv(r".\Data\1950-2022_actual_tornadoes.csv") # Note: we use an 'r' path_name_here because 'r' allows us  to a) copy paste paths directly from your explorer, and mainly b) use '\' without causing error with python

# a good habit is to make  a copy of the original data BEFORE making any changes

orginal_tor_data=tor_data.copy() # now when we want to compare or revert to original data we have a way to do that

# use the itables package with the .show() method
# notice how we can customize the table
show(tor_data.head(), caption="Tornado Data 1950-2022", options={'hover': True})  # itables also allows you to add a title or caption to your table


# caption add a title to the table, it is below the table But still used full for descriptions
# add a hover highlighting function

#  column_filters="header" extension for itables, my personal favorite for exploring table data 



show(tor_data, column_filters="header",layout={"topEnd": None}, maxBytes=0 ,options={'hover': True} )  # Adds individual column filters and removes the single default search bar (which isn't that useful )

# This is similar to an Excel or Google sheets table 
# we added column filters above each column that allows us to visually explore, AND because we set maxBytes=0...
# we can visually search through the entire dataset!!
# note that setting maxBytes=0 consumes a lot of memory for large datasets, as you are loading the full dataset (So, use maxBytes= 0 wisely!)

# add buttons to export the data to your table too!

show(tor_data[(tor_data['yr']==2011)],buttons=["copyHtml5", "csvHtml5", "excelHtml5"], options={'hover': True}) # to fully make use of this we need to filter the data to contain what we want then we can search by  a specific value

show(
    tor_data,
    fixedColumns={"start": 1, "end": 2},
    scrollX=True, options={'hover': True}
)

# this is a neat way to lock columns you want to keep an eye on, and scan across the table

# and for a conditional approach to exploring the data table visually
# use the searchBuilder extension of itables
# this let's you add multiple conditions to filter down the data

show(
    orginal_tor_data,options={'hover': True}
    layout={"top1": "searchBuilder"},maxBytes=0,
    searchBuilder={
        "preDefined": {
            "criteria": [
                {"data": "inj", "condition": "=", "value": [""]}
            ]
        }
    },
)

# Very cool!!
# this is a good way to sanity check and refine any code-based approach
# With this table setup, you can gain a good understanding about what the data is like

# Now that we have a good handle on the data
# let us confirm the type of data we are using 

type(tor_data)

# a dataframe is the technical name for the data table with rows and columns that pandas has created for us

# this is the table of data we read in 
# it has over 68 thousand rows and 29 columns
# first column has no name, BUT that column contains all the row labels of the data, these row labels are called the index

# another way to know how many rows, columns

tor_data.shape

# what if you want to know how many rows, columns, also the data type of each column?

tor_data.info()

# Scanning through, we focus on second to last row, showing 3 data types: float, int, and objects
# that tells you how much of each datatype there is , mostly int64 datatype i.e., integer or whole numbers. 
# Then float or decimal and finally objects which is words

# Any missing data? or non-nulls?,
# Not Surprisingly, we learned that there are no empty rows
    # how do we know? Because all the columns say '68701 non-null' , which is the same number of rows as the entire dataframe
    # there are some caveats to what we define as 'missing' but here we just mean that the row value is blank

# you are the skeptical type, yes? Good...
# see for yourself if we have missing data

tor_data.isnull().sum() # for more on how this works , do this in two steps i.e, tor_data.isnull(), and then print that alone i.e, it is summing true (null=true) and falses for each columns

# why do we care about missing data?
    # it is problem if we want to make fair comparisons
    # it complicates the meaning of the results

# Why care about data type?
    # Pandas has special methods that only apply to certain types of data i.e., dates , times

# let's read in the data again this time let's tell pandas to make the row labels the date

tor_data = pd.read_csv(r".\Data\1950-2022_actual_tornadoes.csv", parse_dates= ['date'])

tor_data.info() 

# check out the date- columns data type now

tor_data['date'].dtypes

#dtype('<M8[ns]') is equivalent to a datetime64 data type in pandas, which is used for date and time data

tor_data.set_index('date') # this shows us the result of the data when it is indexed by date

# # Setting the date as the index is helpful for a time series type anaylsis 
# BUT we did not actually yet change the index to be the date
# this is only a view of the data, it has not really been modified

# if we wanted to make a plot of tornadoes over time we would get an error because we did not change the index of the data to date
# curious to see the error? try this to see an error when tyring to plot the data : tor_data.plot()

# We need to set a special parameter to make sure we actually change the data 
# This is 'inplace' = True parameter to ensure the index is set to date on the tor_data

tor_data.set_index('date', inplace=True) 

# want to see all the columns without using itables?
# Set display options

pd.set_option('display.max_columns', None) # None means show all the columns, but you can use a number to show only that number of columns

tor_data.head() # look at only the top of the data , you can add an arguement into .head() to show that many rows

tor_data.head(20) # add an arguement to see more rows

tor_data.tail() # last only rows of data

# need quick statiscal summary of the data?

tor_data.describe()

# need to see summary for specific columns?
# add double brackets and list your columns in quotes,
# dataframe_name[['column1', 'column2']]

tor_data[["mag", "len", "wid", "loss"]].describe()

# so just using the .describe() method we can access a bunch of statistical summary data for mean, min, max, etc.
# for example, now we can see that the average length a tornado travels across the entire dataset is 3.48 miles!

# only interested in a certain year?
# filter the data by that year
tor_data[(tor_data['yr']==2011)]

#  Say we are interested in injuries in 2011, a year known for having the largest tornado outbreaks i.e., multiple tornado events on the same day
#  So, maybe we should sort by the highest injuries within that known year for tornado outbreaks? 

# filter the data by that year, and then sort by injuries 
tor_data[(tor_data['yr']==2011)].sort_values(by='inj', ascending=False)

# Say within 2011, we are only interested in a certain subset of that year?
# Then filter the data even further. 
# Looking at 2011 tornado events AND those magnitude 2 and above
tor_data[(tor_data['yr']==2011) & (tor_data['mag']>=2)].sort_values(by='inj', ascending=False) # notice how we are chaining together the commands? Filter by year AND mag >=2, then sort injuries highest to lowest

# if we are very interested in that specific date
tor_data.loc[(tor_data.index == '2011-04-27')] # note if the date was not the index we would use: df.loc[(df['date'] == '2022-01-01')]

# Select only a specific row label 
# slice only the first few columns up to mag
tor_data.loc[tor_data.index == '2011-04-27',  :'mag'] # ':' means select all , combined with mag means select all up to and including mag 

# specify the names for only those columns you need
tor_data.loc[tor_data.index == '2011-04-27',  ['st','mag','inj','fat', 'time']] # notice how we are using brackets around the column name list

# if we are very interested in that specific date and the tornadoes occuring with magnitude 2 and up
tor_data.loc[(tor_data.index == '2011-04-27') & (tor_data['mag'] >= 2)] # note if the date was not the index we would use: df.loc[(df['date'] == '2022-01-01') & (df['mag'] >= 2)]

# up until now we were simply filtering the data down to a subset of interest
# and then viewing that subset
# usually what you want is to save that subset of interest into a variable 
# so you can (a) not have to repeat the filtering each time you want to see this subset, (b) focus on it separately from the rest of the data

# if we are very interested in that specific date and the tornadoes occuring with magnitude 2 and up
tor_data_2011_mag2= tor_data.loc[(tor_data.index == '2011-04-27') & (tor_data['mag'] >= 2)] # note if the date was not the index we would use: df.loc[(df['date'] == '2022-01-01') & (df['mag'] >= 2)]

tor_data_2011_mag2.head().sort_values(by="inj", ascending=False) # What is wrong with this?

# the order in which you do things matters...
# The outcome is correct if we sort first, then take a look at the data
tor_data_2011_mag2.sort_values(by="inj", ascending=False).head() # notice the first row is different from above?

# 1500 injuries occurred associated with this tornado event on April 04 2011 in Alabama! 

# Want to understand the data by year?
# So, group your data by year 
# How? using groupby

# 1. groupby is a method that can group by a variable column and within that group count by another column
# 2. here we group by year "yr" column
# 3. within those groups select the "om" column
# 4. Count up the distinct values in the omm column
# 5. call the plot method that is built-into pandas

#            (1)  (2)   (3)    (4)     (5)
tor_data.groupby("yr")["om"].count().plot()

# what did we do here?
# we plotted the data by the year column over time using the builtin pandas plotting .plot()
# it should choose the most logical plot for the data 

# what we are actually plotting is a line plot

tor_data.groupby("yr")["om"].count().plot(kind='line')

# .plot() has different parameters, one is kind, you can set this to 'line' or something else... 

# what if we had a special criteria ?
# only show the data for the years with magnitude greater than or equal to 3

tor_mag_3_data =tor_data[(tor_data["mag"]>=3)] 

tor_mag_3_data.groupby("yr")["om"].count().plot(kind='line')

# plots the mag 3 and up data

tor_mag_3_data.groupby(["st"])["om"].count().sort_values(ascending=False).plot(kind='bar')

# bar plot for the magnitude 3 and up tornadoes grouped by state
# kind is the bar plot
# Options include 'line' (default), 'bar', 'barh', 'hist', 'box', 'kde', 'density', 'area', 'pie', 'scatter', 'hexbin'
# legend: Whether to show the legend. Defaults to True for line plots and False otherwise.

# Always good to cross check your plots in another way
# here we start with the dataset for magnitude 3 and up
# Then we are filtering the tor_mag_3_data for those rows that have state as 'TX' , i.e., texas
 
tor_mag_3_data[tor_mag_3_data["st"]=="TX"].count()

# bar plot results for mag 3 and up:
# Very clear that Texas has the most tornadoes among those with mag 3 and up
# Interesting...
#   But what about overall torndaoes?  Texas still at the top?

# add additional parameters to add labels, change figure size and color etc.
tor_data.groupby("st")["om"].count().sort_values(ascending=False).plot(
    kind='bar', title="Tornado Count by State: 1950-2022", rot=45, legend=True, figsize=(14,8), color='r', grid=False, xlabel="state", ylabel="Tornado counts", fontsize=14)

# Overall, Texas still has the most tornadoes

# A lot that can be customized even using only this built-in pandas plotting function .plot()!
    # pandas is using matplotlib 'under the hood' to generate these plots
# But even more can be done by using matplotlib library directly 
# Or with another libray Seaborn
#  both are specialized for creating very visually sophisticated plots
#... until next time!